package cn.cup.dmp.report.MediaAnalysis

import cn.cup.dmp.config.ConfigHelper
import cn.cup.dmp.utils.MyJedisPool
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * 读取字典文件，将文件内容放入redis中
  */

object PutDictory2Redis {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("读取字典文件").setMaster("local[*]")
    conf.set("spark.serializer", ConfigHelper.ser)
    val sc = new SparkContext(conf)

    //读取文件字典数据，将数据放入redis
    val dictory = sc.textFile("D:\\小牛视频\\资料_广告项目\\app_dict.txt")
    val dict: RDD[(String, String)] = dictory.map(line => {
      line.split("\t", -1)
    }).filter(_.length >= 5).map(arr => (arr(4), arr(1)))
    //将dict 中的数据放入redis中
    dict.foreachPartition(par =>{
      val jedis = MyJedisPool.getRedis()
      par.foreach(t =>{
        jedis.hset("appDictory",t._1,t._2)
      })
      jedis.close()
    })
    sc.stop()
  }

}
